from urllib.robotparser import RobotFileParser
from urllib.request import urlopen

rp = RobotFileParser()

#发生异常: HTTPError
# HTTP Error 418: 
#可能是由于douban的反爬机制，所以目前也没得办法，所以尽量不要用urlopen方法吧
response = urlopen('https://www.douban.com/robots.txt')

rp.parse(response.read().decode('utf-8').split('\n'))



print(rp.can_fetch('Other Spider','https://www.douban.com'))

